package naive2;

import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;

import uk.ac.shef.wit.simmetrics.similaritymetrics.Levenshtein;
import uk.ac.shef.wit.simmetrics.similaritymetrics.MongeElkan;

// bag of words for a document
public class BoW {
	private int totalWords;
	private HashMap<String,Integer> bag;
	public  BoW(){
		this.totalWords=0;
		this.bag = new HashMap<String,Integer>();
	}
	
	
	public void putWord(String word, int freq){
		
		if( freq<=0){
			return;
		}
		
		if(word.equals("")||word.equals(" ")||word.equals("\t")||word.equals("\n"))
			return;
		
		if( this.bag.get(word) == null ){
			//System.out.println("ee");
			//System.out.println(word);
			this.bag.put(word, freq);
		}else{
			int old_freq = bag.get(word);
			this.bag.put(word,old_freq+freq);
		}
	}
	
	public void putWord(String word){
		//bag.containsKey(key)
		//System.out.println("ss");
		
		if(word.equals("")||word.equals(" ")||word.equals("\t")||word.equals("\n"))
			return;
		
		if( this.bag.get(word) == null ){
			//System.out.println("ee");
			//System.out.println(word);
			this.bag.put(word, 1);
		}else{
			int freq = bag.get(word);
			this.bag.put(word,freq+1);
		}
		this.totalWords = this.totalWords+1;
	}
	public int getTotalWords(){
		return this.totalWords;
	}
	
	public int getFreq(String word){
		if ( this.totalWords==0 ){
			return 0;
		}else if( this.bag.get(word) == null ){
			return  0;
		}else{
			return bag.get(word);
		}
	}
	
	public boolean isEmpty(){
		if( this.totalWords==0){
			return true;
		}else{
			return false;
		}
	}
	
	public String toString(){
	
		Set<String> keys = this.bag.keySet();
		String result = "";
		for(String key: keys){
			
			result = result+key+":"+this.bag.get(key)+"\n";
		
		}
		
		
		return result;
	}
	
	public int inCommon(BoW b){
		
		
		Set<String> aset = this.bag.keySet();
		
		Set<String> bset = b.bag.keySet();
		
		Set<String> cset = new HashSet<String>(aset);
		
	
		if( cset.retainAll(bset) ){
			return cset.size();
		}else{
			System.out.println("this is the sub set of the former one, bizzar!");
			return 0;
		}
		
	}

	
	public int inCommonTotal(BoW b){

		Set<String> aset = this.bag.keySet();
		
		Set<String> bset = b.bag.keySet();
		
		Set<String> cset = new HashSet<String>(aset);
		
		if( cset.retainAll(bset) ){
			int score = 0;
			for( String word: cset){
				score = score + b.getFreq(word);
			}
			return score;
		}else{
			return 0;
		}
	}
	
	//print how many word and frequency this have and b have in their common words
	public void printCommon(BoW b){
		Set<String> aset = this.bag.keySet();
		
		Set<String> bset = b.bag.keySet();
		
		Set<String> cset = new HashSet<String>(aset);
		
		cset.retainAll(bset);
		
		for( String key: cset ){
			System.out.println("key:"+key+",this freq:"+this.getFreq(key)+", b freq:"+b.getFreq(key));
		}
		
	}
	
	public float inCommonTotal_normalized(BoW b){
		Set<String> aset = this.bag.keySet();
		
		Set<String> bset = b.bag.keySet();
		
		Set<String> cset = new HashSet<String>(aset);
		
		if( cset.retainAll(bset) ){
			float score = 0;
			for( String word: cset){
				score = score + ((float)b.getFreq(word))/((float)b.getTotalWords());
			}
			return score;
		}else{
			return 0;
		}
	}
	public void printCommon_normalized(BoW b){
		Set<String> aset = this.bag.keySet();
		
		Set<String> bset = b.bag.keySet();
		
		Set<String> cset = new HashSet<String>(aset);
		
		cset.retainAll(bset);
		
		for( String key: cset ){
			float freq1 = ((float)this.getFreq(key))/((float)this.getTotalWords());
			float freq2 = ((float)b.getFreq(key))/((float)b.getTotalWords());
			System.out.println("key:"+key+",this freq:"+freq1+", b freq:"+freq2);
		}
		
	}
	public HashMap<String,Integer> getbag(){
		return this.bag;
	}
	
	//this should be used by doc_bow instead of ebow
	public float tf_idf(BoW idf_count, BoW ebow, int D){
		// get max count in ebow
		float max_count = 0;
		for(int v: ebow.bag.values() ){
			if( v> max_count ){
				max_count = v;
			}
		}
		
		if( max_count==0 ){
			System.out.println("max_count is zero");
			return 0;
		}
		
		//get intersection of words
		Set<String> eset = ebow.bag.keySet();
		Set<String> dset = this.bag.keySet();
		Set<String> cset = new HashSet<String>(dset);
		cset.retainAll(eset);
		
		//tf idf score
		float score = 0;
		for(String word: cset){
			float tf = ((float)ebow.getFreq(word)) / max_count;
			float idf = (float) java.lang.Math.log( ((float)D)/((float)idf_count.getFreq(word)) );
			//System.out.println("word:"+word+",freq:"+ ebow.getFreq(word)+",tf:"+tf+"idf:"+idf);
			score = score + tf*idf;
			
			
		}
		
		return score;
		
	}
	
	//used by doc bow
	public float global_tf_idf(BoW ebow, int D, BoW global_idf){
		// get max count in ebow
		
		float max_count = 0;
		for(int v: ebow.bag.values() ){
			if( v> max_count ){
				max_count = v;
			}
		}
				
		if( max_count==0 ){
			System.out.println("max_count is zero");
			return 0;
		}
		
		System.out.println("max_count for this entity is:"+max_count);
		//get intersection of words
		Set<String> eset = ebow.bag.keySet();
		Set<String> dset = this.bag.keySet();
		Set<String> cset = new HashSet<String>(dset);
		cset.retainAll(eset);
		//tf idf score
		float score = 0;
		for(String word: cset){
			
		
			float tf = ((float)ebow.getFreq(word)) / max_count;
			float idf = (float) java.lang.Math.log( ((float)D)/((float)global_idf.getFreq(word)) );
			//System.out.println("word:"+word+",freq:"+ ebow.getFreq(word)+",tf:"+tf+"idf:"+idf);
			score = score + tf*idf;
			System.out.println("word:"+word+", freq:"+ebow.getFreq(word)+", tf:"+tf+", idf:"+idf+", tf*idf:"+tf*idf+", global Idf freq:"+global_idf.getFreq(word));		
					
		}
		return score;
				
	}
	
	//use nor regularized tf
	public float global_idf_tf1(BoW ebow, int D, BoW global_idf){
		// get max count in ebow
		
		float max_count = 0;
		for(int v: ebow.bag.values() ){
			if( v> max_count ){
				max_count = v;
			}
		}
				
		if( max_count==0 ){
			System.out.println("max_count is zero");
			return 0;
		}
		
		System.out.println("max_count for this entity is:"+max_count);
		//get intersection of words
		Set<String> eset = ebow.bag.keySet();
		Set<String> dset = this.bag.keySet();
		Set<String> cset = new HashSet<String>(dset);
		cset.retainAll(eset);
		//tf idf score
		float score = 0;
		for(String word: cset){
			
		
			//float tf = ((float)ebow.getFreq(word)) / max_count;
			float tf = ebow.getFreq(word);
			float idf = (float) java.lang.Math.log( ((float)D)/((float)global_idf.getFreq(word)) );
			//System.out.println("word:"+word+",freq:"+ ebow.getFreq(word)+",tf:"+tf+"idf:"+idf);
			score = score + tf*idf;
			System.out.println("word:"+word+", freq:"+ebow.getFreq(word)+", tf:"+tf+", idf:"+idf+", tf*idf:"+tf*idf+", global Idf freq:"+global_idf.getFreq(word));		
					
		}
		return score;
	}
	
	public float global_idf_log_tf(BoW ebow, int D, BoW global_idf){
	// get max count in ebow
		
		float max_count = 0;
		for(int v: ebow.bag.values() ){
			if( v> max_count ){
				max_count = v;
			}
		}
				
		if( max_count==0 ){
			System.out.println("max_count is zero");
			return 0;
		}
		
		System.out.println("max_count for this entity is:"+max_count);
		//get intersection of words
		Set<String> eset = ebow.bag.keySet();
		Set<String> dset = this.bag.keySet();
		Set<String> cset = new HashSet<String>(dset);
		cset.retainAll(eset);
		//tf idf score
		float score = 0;
		for(String word: cset){
			
		
			//float tf = ((float)ebow.getFreq(word)) / max_count;
			float tf = tuned_log10((float)ebow.getFreq(word));
			float idf = (float) java.lang.Math.log( ((float)D)/((float)global_idf.getFreq(word)) );
			//System.out.println("word:"+word+",freq:"+ ebow.getFreq(word)+",tf:"+tf+"idf:"+idf);
			score = score + tf*idf;
			System.out.println("word:"+word+", freq:"+ebow.getFreq(word)+", tf:"+tf+", idf:"+idf+", tf*idf:"+tf*idf+", global Idf freq:"+global_idf.getFreq(word));		
					
		}
		return score;
	}
	
	public float tuned_log10(float value){
		if (value<=1){
			value = (float) 1.5;
		}
		return (float) java.lang.Math.log10(value);
	}
	
	//some times , word with high idf only appear ones that make it hard to affect the whole document,
	//this method is to tune this phenomena , that is , if word with high idf only appears less than 5, increase it to 10
	
	public float global_logtf_enhace_idf(BoW ebow, int D, BoW global_idf){
		float idf_threshold = (float)7.0;
		float freq_threshold = (float) 5.0;
		float freq_increase = (float) 10.0;
		float max_count = 0;
		for(int v: ebow.bag.values() ){
			if( v> max_count ){
				max_count = v;
			}
		}
				
		if( max_count==0 ){
			System.out.println("max_count is zero");
			return 0;
		}
		
		System.out.println("max_count for this entity is:"+max_count);
		//get intersection of words
		Set<String> eset = ebow.bag.keySet();
		Set<String> dset = this.bag.keySet();
		Set<String> cset = new HashSet<String>(dset);
		cset.retainAll(eset);
		//tf idf score
		float score = 0;
		for(String word: cset){
			
		
			//float tf = ((float)ebow.getFreq(word)) / max_count;
			float idf = (float) java.lang.Math.log( ((float)D)/((float)global_idf.getFreq(word)) );
			float word_freq = (float) ebow.getFreq(word);
			if( (idf>=idf_threshold)&&(word_freq<=freq_threshold)){
				word_freq = freq_increase;
			}
			
			float tf = tuned_log10(word_freq);
			
			//System.out.println("word:"+word+",freq:"+ ebow.getFreq(word)+",tf:"+tf+"idf:"+idf);
			score = score + tf*idf;
			System.out.println("word:"+word+", freq:"+ebow.getFreq(word)+", tf:"+tf+", idf:"+idf+", tf*idf:"+tf*idf+", global Idf freq:"+global_idf.getFreq(word));		
					
		}
		return score;
	}
	
	
	//based on global_logtf_enhance_idf
	//however, the more idf, the more freq it increases
	public float global_logtf_enhace_idf1(BoW ebow, int D, BoW global_idf){
		float idf_threshold = (float)7.0;
		float freq_threshold = (float) 10.0;
		float increse_base = (float) 10.0;
		float max_count = 0;
		for(int v: ebow.bag.values() ){
			if( v> max_count ){
				max_count = v;
			}
		}
				
		if( max_count==0 ){
			System.out.println("max_count is zero");
			return 0;
		}
		
		System.out.println("max_count for this entity is:"+max_count);
		//get intersection of words
		Set<String> eset = ebow.bag.keySet();
		Set<String> dset = this.bag.keySet();
		Set<String> cset = new HashSet<String>(dset);
		cset.retainAll(eset);
		//tf idf score
		float score = 0;
		for(String word: cset){
			
		
			//float tf = ((float)ebow.getFreq(word)) / max_count;
			float idf = (float) java.lang.Math.log( ((float)D)/((float)global_idf.getFreq(word)) );
			float word_freq = (float) ebow.getFreq(word);
			if( (idf>=idf_threshold)&&(word_freq<=freq_threshold)){
				//word_freq = freq_increase;
				if( idf==7){
					word_freq = word_freq+increse_base;
				}else{
					//word_freq = increse_base(idf-idf_threshold);
					float d = idf-idf_threshold;
					float temp = increse_base;
					for(int i=0;i<d;i++){
						increse_base = increse_base*temp;
					}
					word_freq = increse_base+word_freq;
				}
			}
			
			float tf = tuned_log10(word_freq);
			
			//System.out.println("word:"+word+",freq:"+ ebow.getFreq(word)+",tf:"+tf+"idf:"+idf);
			score = score + tf*idf;
			System.out.println("word:"+word+", freq:"+ebow.getFreq(word)+", tf:"+tf+", idf:"+idf+", tf*idf:"+tf*idf+", global Idf freq:"+global_idf.getFreq(word));		
					
		}
		return score;
	}
	
	//use ME distance
	public float global_logtf_enhace_idf_me(BoW ebow, int D, BoW global_idf){
		float idf_threshold = (float)7.0;
		float freq_threshold = (float) 5.0;
		float freq_increase = (float) 10.0;
		float max_count = 0;
		for(int v: ebow.bag.values() ){
			if( v> max_count ){
				max_count = v;
			}
		}
				
		if( max_count==0 ){
			System.out.println("max_count is zero");
			return 0;
		}
		
		System.out.println("max_count for this entity is:"+max_count);
		//get intersection of words
		//Set<String> eset = ebow.bag.keySet();
		//Set<String> dset = this.bag.keySet();
		//Set<String> cset = new HashSet<String>(dset);
		//cset.retainAll(eset);
		Map<String,Float> meScore= BoW.MECommon(this, ebow);
		//tf idf score
		float score = 0;
		for(String word: meScore.keySet()){
			
			float me_score = meScore.get(word);
			//float tf = ((float)ebow.getFreq(word)) / max_count;
			float idf = (float) java.lang.Math.log( ((float)D)/((float)global_idf.getFreq(word)) );
			//float word_freq = (float) ebow.getFreq(word);
			float word_freq = (float) ebow.getFreq(word)*me_score;
			if( (idf>=idf_threshold)&&(word_freq<=freq_threshold)){
				word_freq = freq_increase*me_score;
			}
			
			float tf = tuned_log10(word_freq);
			
			//System.out.println("word:"+word+",freq:"+ ebow.getFreq(word)+",tf:"+tf+"idf:"+idf);
			score = score + tf*idf;
			System.out.println("word:"+word+"orig_freq:"+ebow.getFreq(word)+", freq:"+word_freq+", tf:"+tf+", idf:"+idf+", tf*idf:"+tf*idf+", global Idf freq:"+global_idf.getFreq(word));		
					
		}
		return score;
	}
	
	//use substring to match
	public static float MEdistance(String s1, String s2){
		Levenshtein le = new Levenshtein();
		float score =  le.getSimilarity(s1, s2);
		if(score<0.5){
			return 0;
		}
		return score;
	}
	
	
	public static Map<String, Float> MECommon(BoW doc, BoW e){
		
		Map<String, Integer> ebag = e.bag;
		Map<String, Integer> dbag = doc.bag;
		Map<String,Float> score = new HashMap<String,Float>();
		for( String s:ebag.keySet()){
			
			float max_score=0;
			for( String s1:dbag.keySet()){
				float temp_score = MEdistance(s,s1);
				if( temp_score>max_score ){
					max_score = temp_score;
				}
			}
			if( max_score>0){
				score.put(s, max_score);
			}
			
		}
		
		return score;
	}
	
	
	public static void main(String[] args){
		//HashMap<String,Integer> map = new HashMap<String,Integer>();
		//if( map.get("a")==null){
		//	System.out.println("true");
		//}
		
		/*BoW bow = new BoW();
		String a = "a";
		String b = "b";
		String c = "c";
		System.out.println("empty?"+bow.isEmpty());
		bow.putWord(a);
		bow.putWord(a);
		bow.putWord(a);
		bow.putWord(c);
		System.out.println("empyt?"+bow.isEmpty());
		System.out.println(bow.getFreq(a));
		System.out.println(bow.getFreq(b));
		System.out.println(bow.getFreq(c));
		System.out.println(bow.getTotalWords());
		System.out.println(bow);
		
		BoW bow1 = new BoW();
		bow1.putWord("a");
		bow1.putWord("a");
		bow1.putWord("e");
		System.out.println(bow1.inCommon(bow));
		System.out.println("above is common,below is total and print common");
		System.out.println(bow1.inCommonTotal(bow));
		
		bow1.printCommon(bow);
		
		System.out.println(bow1);
		bow1.putWord(a,6);
		bow1.putWord("d",4);
		System.out.println(bow1);*/
		/*MongeElkan me = new MongeElkan();
		System.out.println(me.getSimilarity("massa", "masse"));

		System.out.println(me.getSimilarity("mass", "massee"));
	
		Levenshtein le = new Levenshtein();
		System.out.println(le.getSimilarity("massa.", "masse"));

		System.out.println(le.getSimilarity("mass.", "massee"));*/
		
		BoW b1 = new BoW();
		b1.putWord("mass");
		b1.putWord("abe");
		
		BoW b2 = new BoW();
		b2.putWord("mass.");
		b2.putWord("absolutely");
		
		System.out.println(BoW.MECommon( b1,b2));
		
	}
}
